package com.uusee.crawler.fetcher;

import it.unimi.dsi.mg4j.util.MutableString;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.RandomAccessFile;
import java.util.Map;

import org.apache.commons.httpclient.Cookie;
import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.HttpConnectionManager;
import org.apache.commons.httpclient.HttpMethod;
import org.apache.commons.httpclient.HttpStatus;
import org.apache.commons.httpclient.HttpVersion;
import org.apache.commons.httpclient.cookie.CookiePolicy;
import org.apache.commons.httpclient.methods.GetMethod;
import org.apache.commons.httpclient.params.HttpClientParams;
import org.apache.commons.httpclient.params.HttpConnectionManagerParams;
import org.apache.commons.httpclient.params.HttpMethodParams;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;

import com.uusee.crawler.framework.Processor;
import com.uusee.crawler.model.CrawlStatusCodes;
import com.uusee.crawler.model.CrawlURI;
import com.uusee.httpclient.SingleHttpConnectionManager;

public class FetchHTTP extends Processor {
	private static Log log = LogFactory.getLog("FetchHTTP");

	public static final String RANGE = "Range";
	public static final String RANGE_PREFIX = "bytes=0-";
	private static Integer DEFAULT_SOTIMEOUT_MS = new Integer(20000);

	protected void innerProcess(CrawlURI crawlUri) {
		HttpMethod method = null;
		InputStream in = null;
		try {
			String crawlUrl = crawlUri.getCrawlUrl();
			String charset = crawlUri.getCharset();

			// HttpConnectionManager cm = new
			// ThreadLocalHttpConnectionManager();
			HttpConnectionManager cm = new SingleHttpConnectionManager();

			// TODO: The following settings should be made in the corresponding
			// HttpConnectionManager, not here.
			HttpConnectionManagerParams hcmp = cm.getParams();
			hcmp.setConnectionTimeout(DEFAULT_SOTIMEOUT_MS);
			// hcmp.setStaleCheckingEnabled(true);
			// Minimizes bandwidth usage. Setting to true disables Nagle's
			// algorithm. IBM JVMs < 142 give an NPE setting this boolean
			// on ssl sockets.
			hcmp.setTcpNoDelay(false);

			HttpClient http = new HttpClient(cm);
			
			
			HttpClientParams hcp = http.getParams();
			// Set default socket timeout.
			hcp.setSoTimeout(DEFAULT_SOTIMEOUT_MS);
			// Set client to be version 1.0.
			hcp.setVersion(HttpVersion.HTTP_1_0);

			//configureHttpCookies(http,crawlUri);

			// Configure how we want the method to act.
			http.getParams().setParameter(HttpMethodParams.SINGLE_COOKIE_HEADER, new Boolean(true));
			http.getParams().setParameter(HttpMethodParams.UNAMBIGUOUS_STATUS_LINE, new Boolean(false));
			http.getParams().setParameter(HttpMethodParams.STRICT_TRANSFER_ENCODING, new Boolean(false));
			http.getParams().setIntParameter(HttpMethodParams.STATUS_LINE_GARBAGE_LIMIT, 10);

			method = new GetMethod(crawlUrl);
			configureMethod(http, method, crawlUri);
			// method.addRequestHeader("", crawlUrl);
			// method.addRequestHeader("Referer",crawlUrl);
			// http.getParams().setCookiePolicy(CookiePolicy.BROWSER_COMPATIBILITY);
			// method.setRequestHeader("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
			// method.setRequestHeader("Accept-Charset","gb2312,utf-8;q=0.7,*;q=0.7");
			// method.setRequestHeader("Accept-Encoding","gzip,deflate");
			// method.setRequestHeader("Accept-Language","gb2312,utf-8;q=0.7,*;q=0.7");
			// method.setRequestHeader("Connection","Keep-Alive");
			
			// method.setRequestHeader("Host","www.douban.com");
			// method.setRequestHeader("Keep-Alive","300");
			// method.setRequestHeader("Referer","http://www.douban.com/movie/");
			// method.setRequestHeader("User-Agent","Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.9.0.12) Gecko/2009070611 Firefox/3.0.12 (.NET CLR 3.5.30729)");

			int statusCode = http.executeMethod(method);

			if (statusCode == HttpStatus.SC_OK) {
				StringBuilder sb = new StringBuilder();
				in = method.getResponseBodyAsStream();
				BufferedReader fIn = new BufferedReader(new InputStreamReader(in, charset));
				String l = fIn.readLine();
				while (l != null) {
					sb.append(l);
					l = fIn.readLine();
				}
				String crawlResult = sb.toString();
				crawlUri.setCrawlResult(crawlResult);
				crawlUri.setCrawlStatus(CrawlStatusCodes.FETCH_SUCCESS);
			} else {
				log.info(crawlUri.getCrawlUrl() + "----Fetch Failure.StatusCode=" + statusCode);
				crawlUri.setCrawlStatus(CrawlStatusCodes.FETCH_SC_NOT_OK);
			}
		} catch (IOException e) {
			log.error(crawlUri.getCrawlUrl() + "----Fetch Error.", e);
			crawlUri.setCrawlStatus(CrawlStatusCodes.FETCH_EXCEPTION);
		} catch (Exception e) {
			log.error(crawlUri.getCrawlUrl() + "----Fetch Error.", e);
			crawlUri.setCrawlStatus(CrawlStatusCodes.FETCH_EXCEPTION);
		} finally {
			if (method != null) {
				method.releaseConnection();
			}
			if (in != null) {
				try {
					in.close();
				} catch (IOException e) {

				}

			}
		}
	}

	private void configureHttpCookies(HttpClient http, CrawlURI crawlUri) {
		// If Bdb-backed cookies chosen, replace map in HttpState
		if (crawlUri.isCookieFlag()) {
			loadCookies(http, crawlUri.getCookiesFile());
		}
	}

	private static void configureMethod(HttpClient http, HttpMethod method, CrawlURI crawlUri) {
		// Don't auto-follow redirects
		method.setFollowRedirects(false);

		// Use only HTTP/1.0 (to avoid receiving chunked responses)
		method.getParams().setVersion(HttpVersion.HTTP_1_0);
		
		if (crawlUri.isCookieFlag()) {
			method.getParams().setCookiePolicy(CookiePolicy.BROWSER_COMPATIBILITY);
			method.setRequestHeader("Cookie",crawlUri.getCookiesFile());
		}
		
		
		method.setRequestHeader("User-Agent","Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.9.0.12) Gecko/2009070611 Firefox/3.0.12 (.NET CLR 3.5.30729)");
		method.setRequestHeader("From", "master@gmail.com");
		method.addRequestHeader("Content-type", "text/html; charset=" + crawlUri.getCharset());

		// Set retry handler.
		method.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, new CrawlerHttpMethodRetryHandler());

		// final long maxLength = getMaxLength(curi);
		// if(maxLength > 0) {
		// method.addRequestHeader(RANGE,
		// RANGE_PREFIX.concat(Long.toString(maxLength - 1)));
		// }

		// HostConfiguration config = new
		// HostConfiguration(http.getHostConfiguration());
		// configureProxy(curi, config);
		// configureBindAddress(curi, config);
		// return config;
	}

	public void saveCookies(HttpClient http, String saveCookiesFile) {
		// Do nothing if cookiesFile is not specified.
		if (saveCookiesFile == null || saveCookiesFile.length() <= 0) {
			return;
		}

		FileOutputStream out = null;
		try {
			out = new FileOutputStream(new File(saveCookiesFile));
			@SuppressWarnings("unchecked")
			Cookie[] cookies = http.getState().getCookies();
			String tab = "\t";
			out.write("# Heritrix Cookie File\n".getBytes());
			out.write("# This file is the Netscape cookies.txt format\n\n".getBytes());
			for (Cookie cookie : cookies) {
				MutableString line = new MutableString(1024 * 2 /*
																 * Guess an
																 * initial size
																 */);
				line.append(cookie.getDomain());
				line.append(tab);
				line.append(cookie.isDomainAttributeSpecified() == true ? "TRUE" : "FALSE");
				line.append(tab);
				line.append(cookie.getPath());
				line.append(tab);
				line.append(cookie.getSecure() == true ? "TRUE" : "FALSE");
				line.append(tab);
				line.append(cookie.getName());
				line.append(tab);
				line.append((null == cookie.getValue()) ? "" : cookie.getValue());
				line.append("\n");
				out.write(line.toString().getBytes());
			}
		} catch (FileNotFoundException e) {
			// We should probably throw FatalConfigurationException.
			System.out.println("Could not find file: " + saveCookiesFile);
		} catch (IOException e) {
			e.printStackTrace();
		} finally {
			try {
				if (out != null) {
					out.close();
				}
			} catch (IOException e) {
				e.printStackTrace();
			}
		}
	}

	/**
	 * Load cookies from a file before the first fetch.
	 * <p>
	 * The file is a text file in the Netscape's 'cookies.txt' file format.<br>
	 * Example entry of cookies.txt file:<br>
	 * <br>
	 * www.archive.org FALSE / FALSE 1074567117 details-visit texts-cralond<br>
	 * <br>
	 * Each line has 7 tab-separated fields:<br>
	 * <li>1. DOMAIN: The domain that created and have access to the cookie
	 * value.
	 * <li>2. FLAG: A TRUE or FALSE value indicating if hosts within the given
	 * domain can access the cookie value.
	 * <li>3. PATH: The path within the domain that the cookie value is valid
	 * for.
	 * <li>4. SECURE: A TRUE or FALSE value indicating if to use a secure
	 * connection to access the cookie value.
	 * <li>5. EXPIRATION: The expiration time of the cookie value (unix style.)
	 * <li>6. NAME: The name of the cookie value
	 * <li>7. VALUE: The cookie value
	 * 
	 * @param cookiesFile
	 *            file in the Netscape's 'cookies.txt' format.
	 */
	public void loadCookies(HttpClient http, String cookiesFile) {
		// Do nothing if cookiesFile is not specified.
		if (cookiesFile == null || cookiesFile.length() <= 0) {
			return;
		}
		RandomAccessFile raf = null;
		try {
			raf = new RandomAccessFile(cookiesFile, "r");
			String[] cookieParts;
			String line;
			Cookie cookie = null;
			while ((line = raf.readLine()) != null) {
				// Line that starts with # is commented line, therefore skip it.
				if (!line.startsWith("#")) {
					cookieParts = line.split("\\t");
					if (cookieParts.length == 7) {
						// Create cookie with not expiration date (-1 value).
						// TODO: add this as an option.
						cookie = new Cookie(cookieParts[0], cookieParts[5], cookieParts[6], cookieParts[2], -1, Boolean
								.valueOf(cookieParts[3]).booleanValue());

						if (cookieParts[1].toLowerCase().equals("true")) {
							cookie.setDomainAttributeSpecified(true);
						} else {
							cookie.setDomainAttributeSpecified(false);
						}
						http.getState().addCookie(cookie);
						log.fatal("Adding cookie: " + cookie.toExternalForm());
					}
				}
			}
		} catch (FileNotFoundException e) {
			// We should probably throw FatalConfigurationException.
			System.out.println("Could not find file: " + cookiesFile);

		} catch (IOException e) {
			// We should probably throw FatalConfigurationException.
			e.printStackTrace();
		} finally {
			try {
				if (raf != null) {
					raf.close();
				}
			} catch (IOException e) {
				e.printStackTrace();
			}
		}
	}

}
